
import fileinput
import sys
import time
import cPickle

DOMAIN_NAME = 0
CLIENT_IP = 1                  
COMPLETION_TIME = 3    
HTTP_REQUEST = 4       
STATUS_CODE = 5
RETURNED_SIZE = 6
REFERRER = 7
USER_AGENT = 8
dic={}



def hashing():
    start = time.clock()
    count =0
    for line in fileinput.input(['D:\\access_log.13']):
        try:
            if len(line)==0:
                continue
            row = getlogfields(line)
            expandedrow = expandedfields(row)
            if dic.has_key(row[2]):
                ipinfo = dic.get(row[2])
                if ipinfo.has_key(expandedrow[0]):
                    list=ipinfo.get(expandedrow[0])
                    list.append(row[0])
                    count+=1

                else:
                    listip=[]
                    listip.append(row[0])
                    ipinfo[expandedrow[0]]=listip
                    count+=1
            else:
                diction={}
                listip=[]
                listip.append(row[0])
                diction[expandedrow[0]]=listip
                dic[row[2]]=diction
                count+=1
#                print dic.values()
#                break
        except:
            continue
    test()
    
    
#    obj = cPickle.load(open('D:\\save13.p', 'rb'))
#    print "got it back"
#    for key in obj.keys():
#        value=dic.get(key)
#        print value.values()
    elapsed = (time.clock() - start)
    print elapsed
    print count
    sys.exit()
                    

def test():
    print "Success testing "
#    print dic.values()
    if dic.has_key("861ee80996964b5c1a01a8d34a2f31ed"):
        free = dic.get("861ee80996964b5c1a01a8d34a2f31ed")
#        print "Codeeeeeeeeeeee", free.items() 
    cPickle.dump(dic, open('D:\\save13.p', 'wb'))
         
              
  

def getlogfields(s):
    fields = []
    # error! no leading spaces allowed!
    if s.startswith(" "): 
       return fields

    state = 0
    quoteds = ""
    n = 0
    for f in s.split():
        if state == 0:
           if f.startswith("\""):
               if f.endswith("\""):
                   fields.append(f[1:-1])
               else:
                   quoteds = f[1:] + " "
                   state = 2
               continue
           elif f.startswith("'"):
               if f.endswith("\""):
                   fields.append(f[1:-1])
               else:
                   quoteds = f[1:] + " "
                   state = 3
               continue
           elif f.startswith("["):
               if f.endswith("]"):
                   fields.append(f[1:-1])
               else:
                   blockeds = f[1:] + " "
                   state = 4
               continue
           else:
               fields.append(f)
 
        if state == 2 :
            quoteds += (f + " ")
            if f.endswith("\"") :
               quoteds = quoteds[:-2]
               fields.append(quoteds)
               n += 1
               state = 0
 
        if state == 3 :
            quoteds += (f + " ")
            if f.endswith("'") :
               quoteds = quoteds[:-2]
               fields.append(quoteds)
               n += 1
               state = 0
 
        if state == 4 :
            blockeds += (f + " ") 
            if f.endswith("]") :
               blockeds = blockeds[:-2]
               fields.append(blockeds)
               n += 1
               state = 0
 
        #@@@ print "processing [", f, "]; state=", state,";current=",  fields, "quoteds=", quoteds
    return fields

def logsplittimefield(timefield):
    try:
#        print "debugging...", timefield
        (stime, zone) = timefield.split()
        caldate, hour, minute, second = stime[0:].split(":")
        day, month, year = caldate.split("/")
    except:
#        print "error procedding timefield [%s]\n" % timefield
        return None, None, None, None, None, None, None
    return (day, month, year, hour, minute, second, zone)

def logsplitrequestfield(requestfield):
    try:
        # you can avoid the intermediate storing to variables.
        method, resource, protocol = requestfield.split()
    except:
        return None, None, None
 
    return method, resource, protocol
 
 
def logsplituseragentfield(useragentfield):
    """
    Returns (browser, url)
    """
    return useragentfield.split()

def expandedfields(ninefields):
    """
    Returns a full tuple of information.
    """
    try:
        if len(ninefields) == 0:
            return None, None
        ip1, ip2 = ninefields[CLIENT_IP].split(",")
        if ip1 == '-':
            ip = ip2
        else:
            ip = ip1
        day, month, year, hour, minute, second, zone = logsplittimefield(ninefields[COMPLETION_TIME])
        timestamp = year + "-" + getmonth(month) + "-" + day + " " + hour + ":" + minute + ":" + second
        method, resource, protocol = logsplitrequestfield(ninefields[HTTP_REQUEST])
        status = ninefields[STATUS_CODE]
        size = ninefields[RETURNED_SIZE]
        for i, f in enumerate(ninefields):
            print i, f
        browserandurls = ninefields[USER_AGENT].split()
        browser = browserandurls[0]
        if len(browserandurls) == 1:
            urls = []
        else:
            urls = browserandurls[1:]
            urls[0] = urls[0][1:]
            urls[-1] = urls[-1][:-1]

        return (ip, timestamp, day, month, year, hour, minute, second, zone, method, resource,
                protocol, browser, urls)
    except:
#        print "error in processing [%s]" % ninefields
        return None, None, None

def getmonth(month):
    if month == "Jan":
        return "01"
    elif month == "Feb":
        return "02"
    elif month == "Mar":
        return "03"
    elif month == "Apr":
        return "04"
    elif month == "May":
        return "05"
    elif month == "Jun":
        return "06"
    elif month == "Jul":
        return "07"
    elif month == "Aug":
        return "08"
    elif month == "Sep":
        return "09"
    elif month == "Oct":
        return "10"
    elif month == "Nov":
        return "11"
    elif month == "Dec":
        return "12"
    else:
        return "00"

if __name__ == "__main__": hashing()
#import fileinput
#
#DOMAIN_NAME = 0
#CLIENT_IP = 1                  
#COMPLETION_TIME = 3    
#HTTP_REQUEST = 4       
#STATUS_CODE = 5
#RETURNED_SIZE = 6
#REFERRER = 7
#USER_AGENT = 8
#dic={}
#
#
#def hashing():
#
#    for line in fileinput.input(['D:\\format3.txt']):
#        try:
#            if len(line)==0:
#                continue
#            row = getlogfields(line)
#            expandedrow = expandedfields(row)
#            if dic.has_key(row[2]):
#                ipinfo = dic.get(row[2])
#                if ipinfo.has_key(expandedrow[0]):
#                    list=ipinfo.get(expandedrow[0])
#                    list.append(row[0])
#                else:
#                    listip=[]
#                    listip.append(row[0])
#                    ipinfo[expandedrow[0]]=listip
#            else:
#                diction={}
#                listip=[]
#                listip.append(row[0])
#                diction[expandedrow[0]]=listip
#                dic[row[2]]=diction
##                print dic.values()
##                break
#        except:
#            continue
#    test()
#                    
#
#def test():
#    print "Success testing "
#    print dic.values()
#    if dic.has_key("861ee80996964b5c1a01a8d34a2f31ed"):
#        free = dic.get("861ee80996964b5c1a01a8d34a2f31ed")
#        print "Codeeeeeeeeeeee", free.items();
#              
#
##class Logrecord:
##    Hostname, IP, UserId,Time, Request, Response, Downloadvol, Referrer, Useragent
##    def __init__(self):
##        self.hostname = 0
##        self.ip = 0
##        self.userid = 0
##        self.time = 0
##        self.request = 0
##        self.response = 0
##        self.download = 0      
#
#def getlogfields(s):
#    fields = []
#    # error! no leading spaces allowed!
#    if s.startswith(" "): 
#       return fields
#
#    state = 0
#    quoteds = ""
#    n = 0
#    for f in s.split():
#        if state == 0:
#           if f.startswith("\""):
#               if f.endswith("\""):
#                   fields.append(f[1:-1])
#               else:
#                   quoteds = f[1:] + " "
#                   state = 2
#               continue
#           elif f.startswith("'"):
#               if f.endswith("\""):
#                   fields.append(f[1:-1])
#               else:
#                   quoteds = f[1:] + " "
#                   state = 3
#               continue
#           elif f.startswith("["):
#               if f.endswith("]"):
#                   fields.append(f[1:-1])
#               else:
#                   blockeds = f[1:] + " "
#                   state = 4
#               continue
#           else:
#               fields.append(f)
# 
#        if state == 2 :
#            quoteds += (f + " ")
#            if f.endswith("\"") :
#               quoteds = quoteds[:-2]
#               fields.append(quoteds)
#               n += 1
#               state = 0
# 
#        if state == 3 :
#            quoteds += (f + " ")
#            if f.endswith("'") :
#               quoteds = quoteds[:-2]
#               fields.append(quoteds)
#               n += 1
#               state = 0
# 
#        if state == 4 :
#            blockeds += (f + " ") 
#            if f.endswith("]") :
#               blockeds = blockeds[:-2]
#               fields.append(blockeds)
#               n += 1
#               state = 0
# 
#        #@@@ print "processing [", f, "]; state=", state,";current=",  fields, "quoteds=", quoteds
#    return fields
#
#def logsplittimefield(timefield):
#    try:
#        print "debugging...", timefield
#        (stime, zone) = timefield.split()
#        caldate, hour, minute, second = stime[0:].split(":")
#        day, month, year = caldate.split("/")
#    except:
#        print "error procedding timefield [%s]\n" % timefield
#        return None, None, None, None, None, None, None
#    return (day, month, year, hour, minute, second, zone)
#
#def logsplitrequestfield(requestfield):
#    try:
#        # you can avoid the intermediate storing to variables.
#        method, resource, protocol = requestfield.split()
#    except:
#        return None, None, None
# 
#    return method, resource, protocol
# 
# 
#def logsplituseragentfield(useragentfield):
#    """
#    Returns (browser, url)
#    """
#    return useragentfield.split()
#
#def expandedfields(ninefields):
#    """
#    Returns a full tuple of information.
#    """
#    try:
#        if len(ninefields) == 0:
#            return None, None
#        ip1, ip2 = ninefields[CLIENT_IP].split(",")
#        if ip1 == '-':
#            ip = ip2
#        else:
#            ip = ip1
#        day, month, year, hour, minute, second, zone = logsplittimefield(ninefields[COMPLETION_TIME])
#        timestamp = year + "-" + getmonth(month) + "-" + day + " " + hour + ":" + minute + ":" + second
#        method, resource, protocol = logsplitrequestfield(ninefields[HTTP_REQUEST])
#        status = ninefields[STATUS_CODE]
#        size = ninefields[RETURNED_SIZE]
#        for i, f in enumerate(ninefields):
#            print i, f
#        browserandurls = ninefields[USER_AGENT].split()
#        browser = browserandurls[0]
#        if len(browserandurls) == 1:
#            urls = []
#        else:
#            urls = browserandurls[1:]
#            urls[0] = urls[0][1:]
#            urls[-1] = urls[-1][:-1]
#
#        return (ip, timestamp, day, month, year, hour, minute, second, zone, method, resource,
#                protocol, browser, urls)
#    except:
#        print "error in processing [%s]" % ninefields
#        return None, None, None
#
#def getmonth(month):
#    if month == "Jan":
#        return "01"
#    elif month == "Feb":
#        return "02"
#    elif month == "Mar":
#        return "03"
#    elif month == "Apr":
#        return "04"
#    elif month == "May":
#        return "05"
#    elif month == "Jun":
#        return "06"
#    elif month == "Jul":
#        return "07"
#    elif month == "Aug":
#        return "08"
#    elif month == "Sep":
#        return "09"
#    elif month == "Oct":
#        return "10"
#    elif month == "Nov":
#        return "11"
#    elif month == "Dec":
#        return "12"
#    else:
#        return "00"
#
#if __name__ == "__main__": hashing()
